package util;

import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/*这个类作用是处理采集下的网页内容，使用了正则表达式截取出需要采集的网址*/

public class SpiderSougouRobot implements Runnable {

	public void run() {
		final int minIntervalTimer = 10000;
		Connection conn = null;
		Statement stmt = null;
		Connection connCheck = null;
		Statement stmtCheck = null;
		Connection connGetSite = null;
		Statement stmtGetSite = null;
		java.sql.ResultSet rs = null;
		java.sql.ResultSet rsGetSite = null;
		// workingID是个标记位，指示在现在处理到哪一位了
		int workingID = 0;
		String keyword = null;

		try {
			Class.forName("com.mysql.jdbc.Driver");
			conn = DriverManager.getConnection("jdbc:mysql://" + host + "/"
					+ dBName + "?user=root");
			connGetSite = conn;
			connCheck = conn;
			stmtGetSite = connGetSite.createStatement();
			stmtCheck = connCheck.createStatement();
			// 查询出需要处理的网址，check是标记为0就是需要处理的
			rsGetSite = stmtGetSite
					.executeQuery("select * from keywords where sougouSW = 0");
			while (rsGetSite.next()) {
				workingID = rsGetSite.getInt("id");
				String tag = rsGetSite.getString("keyword");
				keyword = URLEncoder.encode(tag, codeNo);
				// 抓取网址开始的时间点
				Long btime = System.currentTimeMillis();
				SpiderBodyXMLSougou sbx = new SpiderBodyXMLSougou();
				sbx.setInterval(interval);
				sbx.setRepeatTimes(repeatTimes);
				sbx.SpiderPageToXML(first + keyword + end);
				String SitesString = sbx.getWebSiteBodyString();
				sbx.clear();
				sbx = null;
				// 到这里已经处理完了当下的wangzhi，需要修改did字段
				stmt = conn.createStatement();
				stmtCheck.execute("update keywords set sougouSW = 1 where id = "
						+ workingID);
				// 如果是0表示链接有错误返回的只是默认值
				if (SitesString == "0") {
					// 出错的没有成功抓取的连接放入badurl表
					stmt.execute("insert into badurl (id,badid) values (0,"
							+ workingID + ")");

				} else {
					// 返回好页面，再抓取其中的链接
					/*//去掉SitesString中的换行和空格
					 * SitesString = SitesString.replaceAll("[\n\r\t]*", "");
					SitesString = SitesString.replaceAll(" ", "");*/
					//System.out.println(SitesString);
					Pattern p = Pattern.compile(urlPattern);
					Matcher m = p.matcher(SitesString);
					int tagURLController = 0;
					while (m.find()) {

						// 这是了重复判断机制，如果有重复的就不加入了，这就使中断后再开始采集时不会出现重复数据
						rs = stmt
								.executeQuery("select id from tagurl where url = '"
										+ m.group(2)
										+ "' and tag = '"
										+ tag
										+ "'");

						if (!rs.next()) {
							//再用一次正则表达式截取出仅包含域名部分的网址，正则的意义是:字母+/就是域名的结束位
							Pattern webSitePattern = Pattern.compile("[a-z]/");
							Matcher webSiteMatcher = webSitePattern.matcher(m.group(2));
							webSiteMatcher.reset();
							//如果能够匹配出主页的网址内容插入数据库，匹配不出就不插入，匹配不出的是ip地址网页质量不会太高
							if (webSiteMatcher.find()) {
								// 字母+/的开始位置再加2个正好就是网址的结束位置
								int j = webSiteMatcher.start() + 2;
								String netPort = new String();
								netPort = m.group(2).substring(0, j);
								stmt.execute("insert into tagurl (id,tag,url,urlnotail) values (0,'" + tag + "','" + m.group(2) + "','" + netPort + "')");
							}
						}
						tagURLController++;
						// 这是一个打印输出，没抓取一个打印出一个数字可以观察程序的运行状态
						System.out.println(tagURLController);
					}
					// tagURLController=0也就是一个链接也没抓取到，把这个页计入到badurl表中
					if (tagURLController == 0) {
						stmt.execute("insert into badurl (id,badid) values (0,"
								+ workingID + ")");
					}
				}
				System.out.println(Thread.currentThread().getName() + "-------" + (workingID));
				Long etime = System.currentTimeMillis();
				System.out.println(etime - btime);
				// 如果不到10秒就加一段延时
				if ((etime - btime) < minIntervalTimer) {
					Thread.sleep(minIntervalTimer - (etime - btime));
				}
			}

		} catch (ClassNotFoundException e) {
			e.printStackTrace();
		} catch (SQLException e) {
			System.out.println("SQLError");
			e.printStackTrace();
		} catch (InterruptedException e) {
			e.printStackTrace();
		} catch (UnsupportedEncodingException e) {
			e.printStackTrace();
		} finally {
			try {
				if (rs != null) {
					rs.close();
					rs = null;
				}
				if (stmt != null) {
					stmt.close();
					stmt = null;
				}
				if (conn != null) {
					conn.close();
					conn = null;
				}
				if (rsGetSite != null) {
					rsGetSite.close();
					rsGetSite = null;
				}
				if (stmtGetSite != null) {
					stmtGetSite.close();
					stmtGetSite = null;
				}
				if (connGetSite != null) {
					connGetSite.close();
					connGetSite = null;
				}
				if (stmtCheck != null) {
					stmtCheck.close();
					stmtCheck = null;
				}
				if (connCheck != null) {
					connCheck.close();
					connCheck = null;
				}
			} catch (SQLException e) {
				e.printStackTrace();
			}
		}
	}

	String urlPattern = null;
	String host = null;
	String dBName = null;
	String first = null;
	String codeNo = null;

	public void setCodeNo(String codeNo) {
		this.codeNo = codeNo;
	}

	public void setFirst(String first) {
		this.first = first;
	}

	String end = null;

	public void setEnd(String end) {
		this.end = end;
	}

	int interval = 0;
	int repeatTimes = 0;

	public void setRepeatTimes(int repeatTimes) {
		this.repeatTimes = repeatTimes;
	}

	public void setInterval(int interval) {
		this.interval = interval;
	}

	public void setHost(String host) {
		this.host = host;
	}

	public void setdBName(String dBName) {
		this.dBName = dBName;
	}

	public void setUrlPattern(String urlPattern) {
		this.urlPattern = urlPattern;
	}
}
